package com.szyjjk.imageCrawler;


import com.szyjjk.entity.SysImageCrawler;
import com.szyjjk.entity.SysImageCrawlerFiles;
import com.szyjjk.service.SysImageCrawlerFilesService;
import com.szyjjk.service.SysImageCrawlerService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;
import java.io.File;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

/**
 * @Author yangyue
 * @Description //TODO 爬虫下载HDF
 * @Date 10:52 2019/4/19
 * @Param
 * @return
 **/
@Component
public class CrawlerFlow {

    /**
     * 下载所需参数
     */
    /****************新版数据流程********************/
    //全局化的爬虫配置信息
    //MOD
    private static Map<String, SysImageCrawler> W_MapModCrawlerInfo = new HashMap<>();
    //Aviso
//    private static SysImageCrawler AvisoCrawlerInfo;
    //Aviso出错次数
    private static int W_erroAviso = 0;
    //Himiwari
//    private static SysImageCrawler HimiwariCrawlerInfo;
    //weget的位置
    private static String WEGETPATH = CrawlerTool.backWgetPath("WgetPath");
    //MODIS  全局信息 当日所下载了的卫星名字
    private static Map<String, List<String>> W_MapDownFileHDFList = new HashMap<>();
    //Aviso 全局信息
    private static List<String> DownFileAvsio = new ArrayList<>();
    //Himiwari 全局信息
    private static String DownFileHimiwari;
    //MYD卫星 全局信息
    private static Map<String, List<String>> W_MAPMYDSatelliteList = new HashMap<>();
    //MOD卫星 全局信息
    private static Map<String, List<String>> W_MAPMODSatelliteList = new HashMap<>();
    //节点时间  转化为毫秒
    // index:0,value: 10:00;
    // index:1,value: 13:00;
    // index:2,value: 16:00;
    // index:3,value: 17:00;
    // index:4,value: 20:00;
    // index:5,value: 23:30;
    //index:6,value:次日10:00
    private static List<Long> NodeDate = new ArrayList<>();

    /*******  数据库服务   ******/
    @Autowired
    private SysImageCrawlerService sysImageCrawlerService;

    @Autowired
    private SysImageCrawlerFilesService sysImageCrawlerFilesService;

//    @Autowired
//    private RunWorkFlowService runWorkFlowService;

    //    @Autowired
//    private RunWorkScheduleService runWorkScheduleService;
    //线程锁
    static Lock lockThread = new ReentrantLock();
    /**
     * 工具类使用服务所需代码
     **/
    public static CrawlerFlow crawlerFlow;

    @PostConstruct
    public void init() {
        crawlerFlow = this;
        this.sysImageCrawlerService = crawlerFlow.sysImageCrawlerService;
        this.sysImageCrawlerFilesService = crawlerFlow.sysImageCrawlerFilesService;
//        this.runWorkFlowService = crawlerFlow.runWorkFlowService;
//        this.runWorkScheduleService = runWorkScheduleService;
    }

    /*****************************    MOD 卫星    **************************************/
    /**
     * MOD 下载程序入口
     *
     * @throws Exception
     */
    public static void crawlerMOD(SysImageCrawler CrawlerInfo) throws Exception {
        //现在执行MOD
        CrawlerTool.systemout("********   MOD(" + CrawlerInfo.getCrawlerRemarks() + ") 任务开始   *********", CrawlerInfo.getFtpRemark());
        //查询MOD 数据
        if (CrawlerInfo == null) {
            CrawlerTool.systemout("当前尚无爬虫数据，请添加数据！", CrawlerInfo.getFtpRemark());
        } else {
            //标识归0 MYD和MOD
            crawlerFlow.sysImageCrawlerService.todayMODTask(CrawlerInfo.getId(), 0);
            crawlerFlow.sysImageCrawlerService.todayMYDTask(CrawlerInfo.getId(), 0);

            //先将标识加一，即使在下载期间下完也没问题，然后在最后时间到了再把这多余的减去
            //设置为全局
//                ModCrawlerInfo = CrawlerInfo;
            W_MapModCrawlerInfo.put(Thread.currentThread().getName(), CrawlerInfo);
            NodeDate = CrawlerTool.getNodeDate();
            CrawlerTool.systemout("   MOD任务启动   ", CrawlerInfo.getFtpRemark());
            //查询已经下载的数据
            List<SysImageCrawlerFiles> todayCrawlerFiles = crawlerFlow.sysImageCrawlerFilesService.findTodayData(CrawlerInfo.getId());
            //存入全局变量
            List<String> hashlist = new ArrayList<>();
            for (SysImageCrawlerFiles crawlerFiles : todayCrawlerFiles) {
//                    System.out.println(Thread.currentThread().getName() + "线程 文件" + crawlerFiles.getFileName());
                CrawlerTool.systemout("   已经用过或者下载完了的   " + crawlerFiles.getFileName(), CrawlerInfo.getFtpRemark());
                hashlist.add(crawlerFiles.getFileName());
            }
            W_MapDownFileHDFList.put(Thread.currentThread().getName(), hashlist);
            //处理下那些卫星 因为有的爬虫下载只下1000m，有的只下250m
            String[] SatelliteArray = CrawlerInfo.getSatellite().split(",");
            CrawlerTool.systemout("   卫星   " + SatelliteArray, CrawlerInfo.getFtpRemark());
            List<String> MODSatelliteList = new ArrayList<>();
            List<String> MYDSatelliteList = new ArrayList<>();
            for (int i = 0; i < SatelliteArray.length; i++) {
                if (SatelliteArray[i].contains("MOD")) {
                    MODSatelliteList.add(SatelliteArray[i]);
                } else if (SatelliteArray[i].contains("MYD")) {
                    MYDSatelliteList.add(SatelliteArray[i]);
                }
            }
            CrawlerTool.systemout("   MOD大小   " + MODSatelliteList.size() + SatelliteArray, CrawlerInfo.getFtpRemark());
            CrawlerTool.systemout("   MYDS大小   " + MODSatelliteList.size() + SatelliteArray, CrawlerInfo.getFtpRemark());
            W_MAPMODSatelliteList.put(Thread.currentThread().getName(), MODSatelliteList);
            W_MAPMYDSatelliteList.put(Thread.currentThread().getName(), MYDSatelliteList);
            //开始进行处理
            judgeDate(CrawlerInfo);
        }
    }

    /**
     * @return void
     * @Author yangyue
     * @Description //TODO 判断时间处于哪个阶段并执行对应的步骤
     * @Date 10:43 2019/7/18
     * @Param []
     **/
    private static void judgeDate(SysImageCrawler sysImageCrawler) throws Exception {
        //Cralwer ID
        String CralerID = W_MapModCrawlerInfo.get(Thread.currentThread().getName()).getId();

        //获取例子名词用于传值
        String mod_Example = "";
        String myd_Example = "";
        List<String> listars = W_MapDownFileHDFList.get(Thread.currentThread().getName());
        if (listars != null) {
            for (String reslutString : listars) {
                if (reslutString.contains("MOD")) {
                    mod_Example = reslutString;
                }
                if (reslutString.contains("MYD")) {
                    myd_Example = reslutString;
                }
            }
        }
        //获取时间
        long nowMile = System.currentTimeMillis();
        //进行对比
        long TenClock = nowMile - NodeDate.get(0);
        if (TenClock < 0) {
            CrawlerTool.systemout("当前时间还不到10:00下载节点" + CrawlerTool.formatDuring(TenClock * -1), sysImageCrawler.getFtpRemark());
            Thread.sleep(TenClock * -1);
            //再次调用
            judgeDate(sysImageCrawler);
        } else {
            long ThirteenClock = nowMile - NodeDate.get(1);
            if (ThirteenClock < 0) {
                CrawlerTool.systemout("处于10:00到13:00之间  进行MOD下载", sysImageCrawler.getFtpRemark());
//                crawlerMain.sysImageCrawlerService.todayTask(ModCrawlerInfo.getId(), 1);
                crawlerFlow.sysImageCrawlerService.todayMODTask(CralerID, 1);
                DwonloadMOD(W_MAPMODSatelliteList.get(Thread.currentThread().getName()), NodeDate.get(1), sysImageCrawler, 1);
            } else {
                //13点之后了
                long fourteenClock = nowMile - NodeDate.get(2);
                if (fourteenClock < 0) {
                    if (mod_Example != "") {
                        tozero(CralerID, mod_Example, -1);
                    }
                    crawlerFlow.sysImageCrawlerService.todayMYDTask(CralerID, 1);
                    CrawlerTool.systemout("处于13:00到16:00之间  进行MYD下载", sysImageCrawler.getFtpRemark());
//                    crawlerMain.sysImageCrawlerService.todayTask(ModCrawlerInfo.getId(), 2);
                    DwonloadMOD(W_MAPMYDSatelliteList.get(Thread.currentThread().getName()), NodeDate.get(2), sysImageCrawler, 1);
                } else {
                    //16点之后了
                    long fiveteenClock = nowMile - NodeDate.get(3);
                    if (fiveteenClock < 0) {
//                        crawlerMain.sysImageCrawlerService.todayTask(ModCrawlerInfo.getId(), 3);
                        if (myd_Example != "") {
                            tozero(CralerID, myd_Example, -1);
                        }
                        CrawlerTool.systemout("处于16:00到17:00之间  等待" + CrawlerTool.formatDuring(fiveteenClock * -1), sysImageCrawler.getFtpRemark());
                        Thread.sleep(fiveteenClock * -1);
                        //然后再次判断
                        judgeDate(sysImageCrawler);
                    } else {
                        //17点之后了
                        long twentyThreeClock = nowMile - NodeDate.get(4);
                        if (twentyThreeClock < 0) {
                            CrawlerTool.systemout("处于17:00到20:00之间  查询数据，若无下载，则开始下载到22:00", sysImageCrawler.getFtpRemark());
                            //首先判断是否下载了MOD
                            int MODnumber = 0;
                            int MYDnumber = 0;
                            List<String> erroList = new ArrayList<>();
                            erroList = W_MapDownFileHDFList.get(Thread.currentThread().getName());
                            if (erroList != null) {
                                for (String StringFile : erroList) {
                                    if (StringFile.contains("MOD")) {
                                        MODnumber++;
                                    }
                                    if (StringFile.contains("MYD")) {
                                        MYDnumber++;
                                    }
                                }
                            }
                            CrawlerTool.systemout("目前MOD已经下载了" + MODnumber + ";MYD已经下载了" + MYDnumber + "个", sysImageCrawler.getFtpRemark());
                            List<String> fiveList = new ArrayList<>();
                            if (MODnumber == 0) {
                                if (mod_Example != "") {
                                    tozero(CralerID, mod_Example, 1);
                                }
                                fiveList.addAll(W_MAPMODSatelliteList.get(Thread.currentThread().getName()));
                            }
                            if (MYDnumber == 0) {
                                if (myd_Example != "") {
                                    tozero(CralerID, myd_Example, 1);
                                }
                                fiveList.addAll(W_MAPMYDSatelliteList.get(Thread.currentThread().getName()));
                            }
                            if (fiveList.size() > 0) {
                                CrawlerTool.systemout("开始重新下载" + fiveList.size() + "个数据", sysImageCrawler.getFtpRemark());
                                DwonloadMOD(fiveList, NodeDate.get(4), sysImageCrawler, 1);
                            } else {
                                CrawlerTool.systemout("当前MOD和MYD 都存在数据，数据检查通过，距离检查时间为" + CrawlerTool.formatDuring(twentyThreeClock * -1), sysImageCrawler.getFtpRemark());
                                Thread.sleep(twentyThreeClock * -1);
                                judgeDate(sysImageCrawler);
                            }
                        } else {
                            long ershisanhreeClock = nowMile - NodeDate.get(5);
                            if (ershisanhreeClock < 0) {
                                SysImageCrawler sysImageCrawlerZero = crawlerFlow.sysImageCrawlerService.findbyID(CralerID);
                                if (sysImageCrawlerZero.getMydMarking() == 1) {
                                    if (myd_Example != "") {
                                        tozero(CralerID, myd_Example, -1);
                                    }
                                } else if (sysImageCrawlerZero.getModMarking() == 1) {
                                    if (mod_Example != "") {
                                        tozero(CralerID, mod_Example, -1);
                                    }
                                }
                                CrawlerTool.systemout("处于20:00到23:30之间 等待到23点距离检查时间为" + CrawlerTool.formatDuring(ershisanhreeClock * -1), sysImageCrawler.getFtpRemark());
                                Thread.sleep(ershisanhreeClock * -1);
                                judgeDate(sysImageCrawler);
                            } else {
                                //23:30之后了
                                CrawlerTool.systemout("时间到23:30，再次进行数据检查，如果有新数据，下载，并删除之前影像信息，", sysImageCrawler.getFtpRemark());
                                String subsping = W_MapModCrawlerInfo.get(Thread.currentThread().getName()).getSatellite();

                                String[] downFilearry = subsping.split(",");
                                //单独执行一下？
                                for (int g = 0; g < downFilearry.length; g++) {
                                    //开始下载数据
                                    int only23 = 0;
                                    List<SysImageCrawlerFiles> MOD02QKMList = finData(downFilearry[g], sysImageCrawler, new Date());


                                    for (SysImageCrawlerFiles ar : MOD02QKMList) {
                                        //截取一下再存

                                         List<String> hdflists = W_MapDownFileHDFList.get(Thread.currentThread().getName());
                                        if (hdflists == null) {
                                            hdflists = new ArrayList<>();
                                        }
                                        if (!hdflists.contains(ar.getFileName())) {
                                            CrawlerTool.systemout("23:30任务  有未下完的数据，开始下载", sysImageCrawler.getFtpRemark());
                                            //存在新的数据，进行删除
                                            if (only23 == 0) {
                                                CrawlerTool.systemout("23:30任务   删除今日影像信息", sysImageCrawler.getFtpRemark());
                                                only23 = 1;
//                                                try {
//                                                    List<RunWorkSchedule> run_work_schedules = crawlerFlow.runWorkScheduleService.getByRunType("绿潮");
//                                                    crawlerFlow.runWorkFlowService.delAllDataByName(run_work_schedules.get(0).getId(), fileNmae);
//                                                } catch (Exception e) {
//                                                    CrawlerTool.systemout(" !!!! 出错 !!! CrakerFlow-303    " + e, sysImageCrawler.getFtpRemark());
//                                                    CrawlerTool.systemout("23:30任务   删除今日影像信息失敗", sysImageCrawler.getFtpRemark());
//                                                    e.printStackTrace();
//                                                }
                                            }
                                            //下载完添加 进入List
                                            List<String> arlist = W_MapDownFileHDFList.get(Thread.currentThread().getName());
                                            CrawlerTool.systemout("23:30任务   需要下载" + arlist.size() + "个任务", sysImageCrawler.getFtpRemark());
                                            arlist.add(ar.getFileName());
                                            W_MapDownFileHDFList.put(Thread.currentThread().getName(), arlist);
                                            //元素不存在，开始下载
                                            ar.setCrawlerDownId(CralerID);
                                            ar.setFileType(sysImageCrawler.getFtpRemark());
                                            MODDownloadThread thread = new MODDownloadThread(sysImageCrawler.getSavePath(), WEGETPATH, ar);
//                                            MODDownloadThread thread = new MODDownloadThread(ar, W_MapModCrawlerInfo.get(Thread.currentThread().getName()).getSavePath(), WEGETPATH, CralerID, W_MapModCrawlerInfo.get(Thread.currentThread().getName()).getCrawlerRemarks());
                                            thread.start();
                                        }
                                    }
                                }
                                //距离第二天的时间
                                long justloveyousomuch = System.currentTimeMillis();
                                long pauseTime = NodeDate.get(6) - justloveyousomuch;
                                CrawlerTool.systemout("今日任务结束，距离下次任务还剩" + CrawlerTool.formatDuring(pauseTime), sysImageCrawler.getFtpRemark());
//                                CrawlerTool.systemout("今日任务结束，", 1);
                            }
                        }
                    }
                }
            }
        }
    }

    //为了在下载标识变为0的之前先清除数据
    private static void tozero(String crawlid, String fileName, int INCR) {
        if (INCR < 0) {
            //减之前查查是否为1----变0之前
            SysImageCrawler sysImageCrawlerZero = crawlerFlow.sysImageCrawlerService.findbyID(crawlid);
//            CrawlerTool.systemout("文件名："+fileName+";当前 MOD 标识符为"+sysImageCrawlerZero.getModMarking(), 1);
//            CrawlerTool.systemout("文件名："+fileName+";当前 MYD 标识符为"+sysImageCrawlerZero.getMydMarking(), 1);
//
//            CrawlerTool.systemout("文件名："+fileName+";标识符开始减一", 1);
//            List<RunWorkSchedule> run_work_schedules = crawlerFlow.runWorkScheduleService.getByRunType("绿潮");
            if (fileName.contains("MOD")) {
                //包含MOD
                if (sysImageCrawlerZero.getModMarking() == 1) {
                    //删除
//                    CrawlerTool.systemout("MOD数据删除，标识符为0", 1);
//                    crawlerFlow.runWorkFlowService.delAllDataByNameOne(run_work_schedules.get(0).getId(), fileName);
                }
            } else if (fileName.contains("MYD")) {
                if (sysImageCrawlerZero.getMydMarking() == 1) {
                    //删除
//                    CrawlerTool.systemout("MYD数据删除，标识符为0", 1);
//                    crawlerFlow.runWorkFlowService.delAllDataByNameOne(run_work_schedules.get(0).getId(), fileName);
                }
            }
//            CrawlerTool.systemout("文件名：" + fileName + ";现在MOD标识符" + (sysImageCrawlerZero.getModMarking() - 1) + ";现在MYD标识符" + (sysImageCrawlerZero.getMydMarking() - 1), sysImageCrawlerInfo.getFtpRemark());
        }
        crawlerFlow.sysImageCrawlerService.updateMasking(crawlid, fileName, INCR);
    }


    /**
     * @return java.lang.String
     * @Author yangyue
     * @Description //TODO 处理数据并下载
     * @Date 10:36 2019/7/18
     * @Param [MODName, indexMod] 【MOD/MYD】【0-5】  indexMod 节点标识
     **/
    public static void DwonloadMOD(List<String> MODName, long indexMod, SysImageCrawler sysImageCrawlerInfo, int soul) throws Exception {
        while (true) {
            long judge = indexMod - System.currentTimeMillis();
            if (judge < 0) {
                //时间在13:00之后
                CrawlerTool.systemout("时间过了节点，" + MODName.get(0) + "等，数据不在继续查询了", sysImageCrawlerInfo.getFtpRemark());
                break;
            } else {
                for (int i = 0; i < MODName.size(); i++) {
                    //开始下载数据
                    findAndDwonLoad(MODName.get(i), sysImageCrawlerInfo, new Date());
                }
            }
            CrawlerTool.systemout("查询等待五分钟！", sysImageCrawlerInfo.getFtpRemark());
            Thread.sleep(300000);
//            Thread.sleep(20000);
        }
        if (soul == 1) {
            judgeDate(sysImageCrawlerInfo);
        } else {
            CrawlerTool.systemout("(MODIS)临时任务完成", sysImageCrawlerInfo.getFtpRemark());
        }

    }

    /*
     * @Author yangyue
     * @Description //TODO 查询一个卫星并下载
     * @Date 9:48 2019/7/18
     * @Param [Stalitin]
     * @return void
     **/
    public static void findAndDwonLoad(String Stalitin, SysImageCrawler sysImageCrawler, Date needDate) throws Exception {
        //Cralwer ID
        String CralerID = sysImageCrawler.getId();
        List<SysImageCrawlerFiles> MOD02QKMList = finData(Stalitin, sysImageCrawler, needDate);
        if (MOD02QKMList == null) {
            CrawlerTool.systemout(Stalitin + "(MODIS)尚未查询到数据！", sysImageCrawler.getFtpRemark());
        } else {
            for (SysImageCrawlerFiles ar : MOD02QKMList) {
                //截取一下再存

                List<String> erroList = new ArrayList<>();
                if (W_MapDownFileHDFList.get(Thread.currentThread().getName()) != null) {
                    erroList = W_MapDownFileHDFList.get(Thread.currentThread().getName());
                }
                if (!erroList.contains(ar.getFileName())) {
                    //开始下载前，添加打卡量，下载完减少一个打卡量，如果打卡为0 则说明全部下完
                    tozero(CralerID, ar.getFileName(), 1);
//                    System.out.println("下载地址" + Thread.currentThread().getName() + "" + W_MapModCrawlerInfo.get(Thread.currentThread().getName()).getSavePath());
                    //元素不存在，开始下载
                    ar.setCrawlerDownId(CralerID);
                    ar.setFileType(sysImageCrawler.getFtpRemark());
                    //下载之前添加 进入List  防止重复下载
                    erroList.add(ar.getFileName());
                    W_MapDownFileHDFList.put(Thread.currentThread().getName(), erroList);

                    MODDownloadThread thread = new MODDownloadThread(sysImageCrawler.getSavePath(), WEGETPATH, ar);
                    thread.start();

                } else {
//                    CrawlerTool.systemout("(MODIS)文件" + ar.getFileName() + "已经下载过了", "Modis");
                    CrawlerTool.systemout(Stalitin + "(MODIS)尚未查询到数据！", sysImageCrawler.getFtpRemark());
                }
            }
        }
    }

    /**
     * @return java.util.List<java.lang.String>
     * @Author yangyue
     * @Description //TODO 查询到数据
     * @Date 15:38 2019/6/25
     * @Param [sataline]
     **/
    private static List<SysImageCrawlerFiles> finData(String sataline, SysImageCrawler sysImageCrawler, Date needDate) throws Exception {
//        System.out.println(sataline);
        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
//        String connUrl = SatelliteURL.get(sataline);
        //四个参数
        String LatLeft = sysImageCrawler.getLatLeft();
        String LonDown = sysImageCrawler.getLonDown();
        String LatRight = sysImageCrawler.getLatRight();
        String LonUp = sysImageCrawler.getLonUp();
//        System.out.println("查询的卫星："+sataline+";"+LatLeft+";"+LonDown+";"+LatRight+";"+LonUp);
        String connUrl = CrawlerTool.NasaDownUrl(sataline, LatLeft, LonDown, LatRight, LonUp, needDate, needDate);
//        Date doDate = new Date();
//        connUrl = connUrl.replaceAll("BEGTIME", simpleDateFormat.format(doDate)).replaceAll("ENDTIME", simpleDateFormat.format(doDate));
        CrawlerTool.systemout("查询的卫星是：" + sataline + "；查询的URL是" + connUrl, sysImageCrawler.getFtpRemark());
        String josnMessage = "";
        try {
            josnMessage = CrawlerTool.doGet(connUrl);
        } catch (Exception e) {
            e.printStackTrace();
            CrawlerTool.systemout(" !!!!(MODIS) 出错 !!! " + e, sysImageCrawler.getFtpRemark());
        }
        if (josnMessage == null || josnMessage.equals("") || josnMessage.length() < 8) {
            return null;
        }
        List<SysImageCrawlerFiles> jsonList = CrawlerTool.NasaJsonStringToList(josnMessage);
        if (jsonList.size() > 0) {
            return jsonList;
        } else {
            return null;
        }
    }

    /**
     * @return void
     * @Author yangyue
     * @Description //TODO 保存数据库信息
     * @Date 10:35 2019/7/22
     * @Param [fileName, downUrl]
     **/
    public static void saveDataMod(SysImageCrawlerFiles sysImageCrawlerFiles) {
        if (sysImageCrawlerFiles.getFileName().contains(".hdf")) {
            tozero(sysImageCrawlerFiles.getCrawlerDownId(), sysImageCrawlerFiles.getFileName(), -1);
        }
        //随机ID
        sysImageCrawlerFiles.setId(CrawlerTool.getRandomChar(30));
        //下载时间
        sysImageCrawlerFiles.setFileDownloadTime(new Date());
        //下载状态
        sysImageCrawlerFiles.setFileState(1);
        //存库
        crawlerFlow.sysImageCrawlerFilesService.saveCrawlerFiles(sysImageCrawlerFiles);
    }


    /*****************************    MOD 卫星END    **************************************/
    /*****************************    Aviso 开始   **************************************/
    public static void crawlerAviso(SysImageCrawler sysImageCrawlerAviso) throws InterruptedException {
        CrawlerTool.systemout("***************    Aviso 开始   ****************", "Aviso");
        //如果文件夹不存在则创建
        if (sysImageCrawlerAviso == null) {
            CrawlerTool.systemout("你没有Aviso数据，请添加数据", "Aviso");
        } else {
            File file = new File(sysImageCrawlerAviso.getSavePath());
            if (!file.exists() && !file.isDirectory()) {
                file.mkdir();
            }
            List<SysImageCrawlerFiles> todayCrawlerFiles = crawlerFlow.sysImageCrawlerFilesService.findTodayData(sysImageCrawlerAviso.getId());
            for (SysImageCrawlerFiles crawlerFiles : todayCrawlerFiles) {
                DownFileAvsio.add(crawlerFiles.getFileName());
            }
            //启动Aviso
            CrawlerTool.systemout("Aviso任务启动", "Aviso");
            findAvido(sysImageCrawlerAviso);
        }
    }

    /**
     * @return void
     * @Author yangyue
     * @Description //TODO 开始查询并下载Avsio 数据
     * @Date 16:39 2019/8/5
     * @Param []
     **/
    public static void findAvido(SysImageCrawler sysImageCrawlerAviso) throws InterruptedException {

        //分出账号 密码
        String name = sysImageCrawlerAviso.getFtpName();
        String pwd = sysImageCrawlerAviso.getFtpPwd();
        String portPath = sysImageCrawlerAviso.getFtpUrl();
        SimpleDateFormat simple = new SimpleDateFormat("/yyyy/MM/");
        List<String> downFileNameList = FTPUtil.getfileList(sysImageCrawlerAviso.getDownloadUrl(), portPath + simple.format(new Date()), name, pwd, sysImageCrawlerAviso.getSatellite(), 1);

        if (downFileNameList == null) {
            CrawlerTool.systemout("（Avsio）查询不到数据等待五分钟再次查询", "Aviso");
            Thread.sleep(30000);
            findAvido(sysImageCrawlerAviso);
            return;
        }
        try {
            for (String fileNmae : downFileNameList) {
                if (!DownFileAvsio.contains(fileNmae)) {
                    CrawlerTool.systemout("（Avsio）需要下载" + fileNmae, "Aviso");
                    String downulr = sysImageCrawlerAviso.getDownloadUrl() + portPath + simple.format(new Date()) + fileNmae;
                    SysImageCrawlerFiles sysImageCrawlerFiles = CrawlerTool.wgetDown(downulr, sysImageCrawlerAviso.getSavePath(), fileNmae, WEGETPATH, name, pwd, "Aviso");

                    if (sysImageCrawlerFiles != null) {
                        if (!sysImageCrawlerFiles.getFileSize().equals("0")) {
//                            CrawlerTool.systemout(fileNmae + "下载耗时" + CrawlerTool.formatDuring(Long.parseLong(file[0])), "Aviso");
                            sysImageCrawlerFiles.setCrawlerDownId(sysImageCrawlerAviso.getId());
                            saveDataMod(sysImageCrawlerFiles);
                        }
                    }
                } else {
                    CrawlerTool.systemout(fileNmae + "文件已下载", "Aviso");
                }
            }
        } catch (Exception e) {
            CrawlerTool.systemout("    出错  " + e, "Aviso");
            CrawlerTool.systemout(" 获取文件出错，异常捕获，正在重新下载", "Aviso");
            Thread.sleep(30000);
            findAvido(sysImageCrawlerAviso);
            e.printStackTrace();
            return;
        }
        //添加一个判断，如果走到这今天还没下到数据，那就循环查询接着下载   ---预防措施
        List<SysImageCrawlerFiles> listaller = crawlerFlow.sysImageCrawlerFilesService.findTodayData(sysImageCrawlerAviso.getId());
        if (listaller == null || listaller.size() == 0) {
            CrawlerTool.systemout(" 当日尚未下载数据,开始重新下载", "Aviso");
            W_erroAviso++;
            if (W_erroAviso > 50) {
                CrawlerTool.systemout(" 当日尚未下载数据,下载出错次数满足，不在继续下载", "Aviso");
            } else {
                Thread.sleep(30000);
                findAvido(sysImageCrawlerAviso);
            }
        }
        CrawlerTool.systemout("当日下载任务已结束", "Aviso");
    }


/*****************************    Aviso END   **************************************/

//    目前废弃

    /*****************************    Himiwari     **************************************/
    public static void crawlerHimiwari(SysImageCrawler CrawlerInfo) throws InterruptedException {
        CrawlerTool.systemout("***************    Himiwari 开始   ****************", "Himiwari");
        //如果文件夹不存在则创建
        if (CrawlerInfo == null) {
            CrawlerTool.systemout("你没有Himiwari数据，请添加数据", "Himiwari");
        } else {
            File file = new File(CrawlerInfo.getSavePath());
            if (!file.exists() && !file.isDirectory()) {
                file.mkdir();
            }
            //全局化
            List<SysImageCrawlerFiles> todayCrawlerFiles = crawlerFlow.sysImageCrawlerFilesService.findTodayData(CrawlerInfo.getId());
            //存入全局变量
            if (todayCrawlerFiles.size() > 0) {
                DownFileHimiwari = todayCrawlerFiles.get(0).getFileName();
                CrawlerTool.systemout(DownFileHimiwari + "为最后一个下载的文件(Himiwari)", "Himiwari");
            }
            //启动Aviso
            CrawlerTool.systemout("Himiwari任务启动.24小时查询最新数据进行下载", "Himiwari");
            findHimiwariName(CrawlerInfo);
        }
    }


    //开始查询数据
    public static void findHimiwariName(SysImageCrawler HimiwariCrawlerInfo) throws InterruptedException {

        //分出账号 密码
        String name = HimiwariCrawlerInfo.getFtpName();
        String pwd = HimiwariCrawlerInfo.getFtpPwd();
        String portPath = HimiwariCrawlerInfo.getFtpUrl();
        SimpleDateFormat simple = new SimpleDateFormat("/yyyyMM/dd/");
        Calendar calendar = Calendar.getInstance();
        calendar.setTime(new Date());
        calendar.set(Calendar.HOUR, calendar.get(Calendar.HOUR) - 8);
        String donwtime = simple.format(calendar.getTime());
        List<String> downFileNameList = new ArrayList<>();
        try {
            downFileNameList = FTPUtil.getfileList(HimiwariCrawlerInfo.getDownloadUrl(), portPath + donwtime, name, pwd, HimiwariCrawlerInfo.getSatellite(), 2);
        } catch (Exception e) {
            //出错
            e.printStackTrace();
            errosout(HimiwariCrawlerInfo);
        }
        String downFilename = "";
        if (downFileNameList == null || downFileNameList.size() <= 0) {
            CrawlerTool.systemout("Himiwari 查询不到数据，等待再次查询", "Himiwari");
            Thread.sleep(30000);
            findHimiwariName(HimiwariCrawlerInfo);
            return;
        } else {
            downFilename = downFileNameList.get(downFileNameList.size() - 1);
        }

        if (downFilename.equals(DownFileHimiwari)) {
            errosout(HimiwariCrawlerInfo);
        } else {
            //开始下载
            CrawlerTool.systemout("(Himiwari)需要下载" + downFilename, "Himiwari");
            String downulr = HimiwariCrawlerInfo.getDownloadUrl() + portPath + simple.format(calendar.getTime()) + downFilename;
            SysImageCrawlerFiles sysImageCrawlerFiles = CrawlerTool.wgetDown(downulr, HimiwariCrawlerInfo.getSavePath(), downFilename, WEGETPATH, name, pwd, "Himiwari");

            Thread.sleep(2000);
            DownFileHimiwari = downFilename;
//            String[] fileresutt = downolMile.split(",");
            if (!sysImageCrawlerFiles.getFileSize().equals("0")) {
//                CrawlerTool.systemout("(Himiwari)" + downFilename + "下载耗时" + CrawlerTool.formatDuring(Long.parseLong(fileresutt[0])), "Himiwari");
                sysImageCrawlerFiles.setCrawlerDownId(HimiwariCrawlerInfo.getId());
                saveDataMod(sysImageCrawlerFiles);
            }
        }
        CrawlerTool.systemout("(Himiwari)" + downFilename + "已完成", "Himiwari");
    }


    /**
     * @return void
     * @Author yangyue
     * @Description //TODO  Himwari Error輸出
     * @Date 16:53 2019/8/23
     * @Param []
     **/
    private static void errosout(SysImageCrawler HimiwariCrawlerInfo) {
        long tingzhi = HimiwariCrawlerInfo.getCrawlerInterval() * 1000 * 60;
        CrawlerTool.systemout("(Himiwari)查询失败，等待一会", "Himiwari");
        try {
            Thread.sleep(tingzhi);
            findHimiwariName(HimiwariCrawlerInfo);
        } catch (InterruptedException e) {
            CrawlerTool.systemout("(Himiwari) errosout 出错   " + e, "Himiwari");
            e.printStackTrace();
        }

    }

    /*******************   其他数据（目前包含Himwari(FTP)、H2B数据）   *****************************************/

    /**
     * @return void
     * @Author yangyue
     * @Description //TODO    其他数据（目前包含Himwari(FTP)、H2B数据）
     * @Date 14:48 2019/9/17
     * @Param [sysImageCrawler]
     **/
    public static void beginElse(SysImageCrawler sysImageCrawler) {
        CrawlerTool.systemout("***************" + sysImageCrawler.getSatellite() + " ****************", sysImageCrawler.getSatellite());
        //如果文件夹不存在则创建
        File file = new File(sysImageCrawler.getSavePath());
        if (!file.exists() && !file.isDirectory()) {
            file.mkdir();
        }
        //启动
        CrawlerTool.systemout(sysImageCrawler.getSatellite() + "任务启动", sysImageCrawler.getSatellite());
        findH2bDate(sysImageCrawler);
    }


    /**
     * @return void
     * @Author yangyue
     * @Description //TODO 开始查询数据并下载
     * @Date 14:17 2019/12/6
     * @Param [imageCrawler]
     **/
    private static void findH2bDate(SysImageCrawler crawler) {
        lockThread.lock();
        //数据处理要求
        //1. 查询出所有数据，返回最新的数据
        //2.筛选出Temp 以及 不需要的波段（暂时未知）
        try {
            List<String> ALTList = FTPUtil.getListH2B(crawler);
            //查询出所有
            for (String arName : ALTList) {
                //便利出所有需要进行下载的文件夹
                if (crawlerFlow.sysImageCrawlerFilesService.getCrawlerFileByFileName(crawler.getId(), arName).size() == 0) {
                    //数据库不存在 开始下载。
                    CrawlerTool.systemout("需要下载" + arName, crawler.getSatellite());
                    //处理下载链接
                    String downulr = "";
                    if (crawler.getFtpUrl() != null) {
                        //东西为空就说明中间要加一个时间  有序的，这个就便利了一下一级文件夹
                        downulr = crawler.getDownloadUrl() + new SimpleDateFormat(crawler.getFtpUrl()).format(CrawlerTool.setTime(new Date(), 0, 0, 0, -8, 0)) + "/" + arName;
                    } else {
                        //没有就说明遍历了所有文件夹
                        if (arName.contains("/")) {
                            //如果有斜杠就说明有存在文件下文件夹
                            arName = arName.substring(arName.lastIndexOf("/"), arName.length());
                        }
                        downulr = crawler.getDownloadUrl() + "/" + arName;
                    }
                    String savePath = crawler.getSavePath() + "\\" + new SimpleDateFormat("yyyyMMdd\\").format(new Date());
                    SysImageCrawlerFiles sysImageCrawlerFiles = CrawlerTool.wgetDown(downulr, savePath, arName, WEGETPATH, crawler.getFtpName(), crawler.getFtpPwd(), crawler.getSatellite());
                    //存库

                    if (!sysImageCrawlerFiles.getFileSize().equals("0")) {
//                        CrawlerTool.systemout(arName + "下载耗时" + CrawlerTool.formatDuring(Long.parseLong(fileresutt[0])), crawler.getSatellite());
                        sysImageCrawlerFiles.setCrawlerDownId(crawler.getId());
                        saveDataMod(sysImageCrawlerFiles);
                    }
                } else {
                    CrawlerTool.systemout("已经下载" + arName, crawler.getSatellite());
                }
            }
        } catch (Exception e) {
            CrawlerTool.systemout(e.toString(), crawler.getSatellite());
        } finally {
            lockThread.unlock();
        }
    }


}


